<?php
//引入公共文件
require('./config.php');

$action = trim($_GET['action']);
if( $action == 'addRule'){
	saveRule();
}else if( $action == 'deleteRule'){
	deleteRule();
}else if( $action == 'saveRule'){
	saveRule();
}else if( $action == 'gather'){
	gather();//采集
}else if( $action == 'export'){
	export();//导出
}else if ( $action == 'import'){
	import(); //导入
}else{
	echo 'other';
}

die('<hr/>');

/**
 * 采集
 *
 */
function gather(){
	set_time_limit(0);
	global $conn;
	global $tabPre;
	print str_repeat(' ',5000);
	$id  = intval($_GET['id']);
	$sql = "SELECT * FROM {$tabPre}gather_url WHERE id='$id' LIMIT 1;";
	$res = mysql_query($sql,$conn);
	$row = mysql_fetch_array($res);

	$startPage        = $row['startPage'];
	$endPage          = $row['endPage'];
	$tempGatherUrl    = $row['gatherUrl'];
	$subjectPat       = $row['subjectPat'];
	$listUrlSeq       = $row['listUrlSeq'];
	$gatherWebCharset = $row['gatherWebCharset'];
	$contentPat       = $row['contentPat'];
	$subjectSeq       = $row['subjectSeq'];
	$contentSeq       = $row['contentSeq'];

	$gatherUrlType    = $row['gatherUrlType'];
	$gatherUrlSets    = $row['gatherUrlSets'];
	$firstPageUrl     = $row['firstPageUrl'];

	$subjectReplace   = $row['subjectReplace'];
	$contentReplace   = $row['contentReplace'];


	$subjectUrlMustInclude = $row['subjectUrlMustInclude'];
	$subjectUrlNotInclude  = $row['subjectUrlNotInclude'];
	$contentUrlMustInclude = $row['contentUrlMustInclude'];
	$contentUrlNotInclude  = $row['contentUrlNotInclude'];
	
	$subjectMustInclude = $row['subjectMustInclude'];
	$subjectNotInclude  = $row['subjectNotInclude']; 
	$contentMustInclude = $row['contentMustInclude'];  
	$contentNotInclude  = $row['contentNotInclude'];  
	$subjectUseRealPat  = $row['subjectUseRealPat']; 
	$subjectRealPat     = $row['subjectRealPat'];  
	$contentUseRealPat  = $row['contentUseRealPat'];  
	$contentRealPat     = $row['contentRealPat'];  
	$remotePicToLocal   = $row['remotePicToLocal']; //是否本地化图片 
	$dbInfo             = unserialize($row['dbInfo']); //数据库连接信息

	$test = isset($_GET['test']) ? trim($_GET['test']) : '' ;
	if($test!='test'){ //采集
		if($dbInfo){
			$dbLink  = explode(',', $dbInfo['link']);
			$hostArr = explode(':', $dbLink[0]);
			$host    = $hostArr[1];
			$userArr = explode(':', $dbLink[1]);
			$user    = $userArr[1];
			$passArr = explode(':', $dbLink[2]);
			$pass    = $passArr[1];
			$dbArr   = explode(':', $dbLink[3]);
			$db      = $dbArr[1];
			$table   = explode(':', $dbLink[4]);
			$table   = $table[1];

			$fileds  = explode(',', $dbInfo['field']);
			$key     = '';
			$value   = '';
			foreach ($fileds as $f){
				$ff = explode(':', $f);
				$key .= ','.$ff[0]; //字段
				if($ff[1]=='time'){
					$value .=",".time();
				}else{
					$value .= ",'".$ff[1]."'"; //值
				}
			}
			$key   = substr($key, 1);
			$value = substr($value, 1);


			//连接数据库
			p( "正在连接数据库","<br />");
			$targetLink = mysql_connect($host,$user,$pass) or die("不能连接目标数据库".mysql_error());
			p( "连接数据库成功！正在选择数据库...","<br />");
			mysql_select_db($db,$targetLink) or die("不能选择数据库".mysql_error());
			mysql_query('set names utf8',$targetLink);			
			p( "连接数据库成功,开始采集....","<br />");
		}
	}

	
	$listUrlAll = array(); //待采集的网址
	if($gatherUrlType==0){ //网址类型 为 连续的
		$totalPage  = $endPage - $startPage + 1;
		$articleSeq = 0;
		p("一共有{$totalPage}({$startPage}-{$endPage})页需要采集",'<br/>');
		 
		for($page=$startPage; $page<=$endPage; $page++){ 
			$listUrlAll[$page] = str_replace('[--page--]',$page,$tempGatherUrl);
			if($page==1 && $firstPageUrl!='' ){//第一页不一样
				$listUrlAll[$page] = $firstPageUrl;
			}
		}
	}else{ //网址不连续
		$listUrlAll = explode("\r\n",$gatherUrlSets);
		$totalPage  = sizeof($listUrlAll);
		$articleSeq = 0;
		p("一共有{$totalPage}页需要采集",'<br/>');
	}
 
	
	//采集开始
	foreach($listUrlAll as $gatherUrl){
		if($gatherUrl==''){
			continue;
		}
		if($listContent = gc($gatherUrl,$gatherWebCharset)){
			if($subjectUseRealPat==0){ //标题使用简易规则
				$pat = getPat($subjectPat);
				preg_match_all("/$pat/is",$listContent,$matchList);
			}else{
				preg_match_all($subjectRealPat,$listContent,$matchList);
			}
			$size = sizeof($matchList[0]);
			for($i=0;$i<$size;$i++){
				$article = array();
				$article['subject'] = $matchList[$subjectSeq][$i];
				$article['href']    = getRealUrl($gatherUrl,$matchList[$listUrlSeq][$i]);
 
				if($subjectUrlMustInclude!=''){ //标题url必须包含指定字符
					if(strpos($article['href'],$subjectUrlMustInclude)===false){
						continue;
					}
				}
				if($subjectUrlNotInclude!=''){ //标题url不能出现指定字符
					if(strpos($article['href'],$subjectUrlNotInclude)!==false){
						continue;
					}
				}
				if($contentUrlMustInclude!=''){ //内容url必须包含指定字符
					if(strpos($article['href'],$contentUrlMustInclude)===false){
						continue;
					}
				}
				if($contentUrlNotInclude!=''){ //内容url不能出现指定字符
					if(strpos($article['href'],$contentUrlNotInclude)!==false){
						continue;
					}
				}
				if($subjectMustInclude!=''){ //标题必须包含指定字符
					if(strpos($article['subject'],$subjectMustInclude)===false){
						continue;
					}
				}
				if($subjectNotInclude!=''){ //标题不能出现指定字符
					if(strpos($article['subject'],$subjectNotInclude)!==false){
						continue;
					}
				}
				if(!empty($subjectReplace)){
					$article['subject'] = getReplace($article['subject'], $subjectReplace);
				}
				
 
				$cnt             = 0;
				$detailContent   = FALSE;
				while($cnt < 3 && ($detailContent=gc($article['href'],$gatherWebCharset))===FALSE ) 
				$cnt++;
				++$articleSeq;
	 
				if($detailContent){
		 
					if($contentUseRealPat==0){ // 详情使用简易规则
						$patContent = getPat($contentPat);
						preg_match("~".$patContent."~isU",$detailContent,$matchContent);
					}else{
						preg_match($contentRealPat,$detailContent,$matchContent);
					}
					
					$article['content'] = ds(filterStr($matchContent[$contentSeq]));
					if(!empty($contentReplace)){
						$article['content'] = getReplace($article['content'], $contentReplace);
					}
 
					if($contentMustInclude!=''){ //内容必须包含指定字符
						if(strpos($article['content'],$contentMustInclude)===false){
							continue;
						}
					}
					if($contentNotInclude!=''){ //内容不能出现指定字符
						if(strpos($article['content'],$contentNotInclude)!==false){
							continue;
						}
					}
					
					$gatherHost = getDomain($article['href']); //取得采集的网站域名
					// 替换路径如 src="/upload/**/**/xx.jpg" 的图片地址 
					$rp                 = "src=$gatherHost\\2 ";
					$article['content'] = preg_replace("/(src|SRC)=[\"|'| ]{0,}(\/(.*)\.(gif|jpg|jpeg|bmp|png))/isU",$rp,$article['content']);
					
					if($test !='test' ){
						if($remotePicToLocal==1){ //本地化图片
							$article['content'] = getPic($article['content'], "./pic");
						}
					}
					
					
					$article['subject'] = strip_tags($article['subject']); //防止标题有图片，比如最新的 精华的等
 
					if($test !='test' && $targetLink){ //非测试且数据库连接正常
						if(trim($article['content']) != ''){
							$ivalue = str_replace('title', "$article[subject]", $value); //替换标题
							$ivalue = str_replace('content', "$article[content]", $ivalue); //替换正文
							$sql    = "insert into {$table} ($key) values ($ivalue)";
							mysql_query($sql,$targetLink) or die($sql ); //插入数据库
						} else {
							p("<div style='border:1px solid #ccc;margin-top:10px;'><h2>第<font color=red>{$articleSeq}</font>篇文章</h2>标题：{$article['subject']}<br/>网址：{$article['href']}<br/><font color=red>获取文章成功，但是匹配失败</font></div>"," ");	
							die($article['content']);
							continue;
						}
						unset($ivalue);
					}
					$message = "<div style='border:1px solid #ccc;margin-top:10px;'><h2>第<font color=red>{$articleSeq}</font>篇文章</h2>标题：{$article['subject']}<br/>网址：{$article['href']}</div>";
					p($message," ");
				}else{
					p("<div style='border:1px solid #ccc;margin-top:10px;'><h2>第<font color=red>{$articleSeq}</font>篇文章</h2>标题：{$article['subject']}<br/>网址：{$article['href']}<br/>正文：<font color=red>获取文章失败</font></div>"," ");
				}
			}
		}else{
			p('<font color=yellow>不能获取网址的<a href="'.$gatherUrl.'" target="_blank" >'.$gatherUrl.'</a>内容</font>');
		}
	}
}


/**
 * 保存规则
 *
 */
function saveRule(){
	global $conn;
	global $tabPre;
	$id               = intval($_POST['id']);
	$gatherWebName    = trim($_POST['gatherWebName']);
	$gatherWebCharset = trim($_POST['gatherWebCharset']);
	$gatherUrlType    = intval($_POST['gatherUrlType']);
	$gatherUrl        = trim($_POST['gatherUrl']);
	$startPage        = intval($_POST['startPage']);
	$endPage          = intval($_POST['endPage']);
	$gatherUrlSets    = trim(trim($_POST['gatherUrlSets']),"\r\n");
	$firstPageUrl     = trim($_POST['firstPageUrl']);
	$subjectPat       = trim($_POST['subjectPat']);
	$contentPat       = trim($_POST['contentPat']);
 
	$subjectMustInclude = trim($_POST['subjectMustInclude']); 
	$subjectNotInclude  = trim($_POST['subjectNotInclude']);
	$contentMustInclude = trim($_POST['contentMustInclude']);
	$contentNotInclude  = trim($_POST['contentNotInclude']);
	$subjectUseRealPat  = trim($_POST['subjectUseRealPat']);
	$subjectRealPat     = trim($_POST['subjectRealPat']);
	$contentUseRealPat  = trim($_POST['contentUseRealPat']);
	$contentRealPat     = trim($_POST['contentRealPat']);
 
	$subjectSeq       = intval($_POST['subjectSeq']);
	$listUrlSeq       = intval($_POST['listUrlSeq']);
	$contentSeq       = intval($_POST['contentSeq']);
	$subjectReplace   = trim($_POST['subjectReplace']);
	$contentReplace   = trim($_POST['contentReplace']);

	$remotePicToLocal      = intval($_POST['remotePicToLocal']);
	$allowSameSubject      = intval($_POST['allowSameSubject']);
	$subjectUrlMustInclude = trim($_POST['subjectUrlMustInclude']);
	$subjectUrlNotInclude  = trim($_POST['subjectUrlNotInclude']);
	$contentUrlMustInclude = trim($_POST['contentUrlMustInclude']);
	$contentUrlNotInclude  = trim($_POST['contentUrlNotInclude']);

	//数据库连接信息
	$dbLink          = trim($_POST['dbLink']);
	$dbField         = trim($_POST['dbField']);
	$dbInfo['link']  = $dbLink;
	$dbInfo['field'] = $dbField;
	$dbInfo          = serialize($dbInfo);
 

	if($gatherWebName=='' || $gatherWebCharset==''  || $subjectSeq==0 || $listUrlSeq==0 || $contentSeq ==0){
		die('网站名称,网站编码和规则不能为空,标题序号，链接序号，内容序号至少为1');
	}
	if($gatherUrlType==0){
		if($gatherUrl=='' || $startPage==0 || $endPage==0 ){
			die('网址类型 为 “连续的” 时，采集网址，开始页数，结束页数 都不能为空~');
		}
	}else{
		if($gatherUrlSets==''){
			die('网址类型 为 “指定的” 时，指定页面 不能为空~');
		}
	}
	if($subjectUseRealPat==0){ //标题使用简易正则
		if($subjectPat == ''){
			die('标题规则 不能为空~');
		}
	} else { //标题使用php正则
		if($subjectRealPat == ''){
			die('标题正则 不能为空~');
		}
	}
	
	if($contentUseRealPat==0){ //详情使用简易正则
		if($contentPat == ''){
			die('详情规则 不能为空~');
		}
	} else { //详情使用php正则
		if($contentRealPat == ''){
			die('详情正则 不能为空~');
		}
	}

 

	if($id){ //更新
		$sql = "UPDATE {$tabPre}gather_url SET `gatherWebName`='$gatherWebName' ,`gatherWebCharset`='$gatherWebCharset',`gatherUrlType`='$gatherUrlType',`gatherUrlSets`='$gatherUrlSets',`firstPageUrl`='$firstPageUrl', `gatherUrl`='$gatherUrl',`startPage`='$startPage',`endPage`='$endPage',`subjectSeq`='$subjectSeq',`listUrlSeq`='$listUrlSeq',`contentSeq`='$contentSeq',`subjectPat`='$subjectPat',`contentPat`='$contentPat' ,`subjectReplace`='$subjectReplace', `contentReplace`='$contentReplace' ,`remotePicToLocal`='$remotePicToLocal', `allowSameSubject`='$allowSameSubject', `subjectUrlMustInclude`='$subjectUrlMustInclude', `subjectUrlNotInclude`='$subjectUrlNotInclude', `contentUrlMustInclude` ='$contentUrlMustInclude', `contentUrlNotInclude`='$contentUrlNotInclude',`dbInfo`='$dbInfo', `subjectMustInclude` = '$subjectMustInclude', `subjectNotInclude` = '$subjectNotInclude', `contentMustInclude` = '$contentMustInclude', `contentNotInclude` = '$contentNotInclude', `subjectUseRealPat` = '$subjectUseRealPat',  `subjectRealPat` = '$subjectRealPat', `contentUseRealPat` = '$contentUseRealPat', `contentRealPat` = '$contentRealPat' WHERE id='$id' LIMIT 1;";
		if(mysql_query($sql,$conn)){
			echo "<script>alert('修改成功~');window.location='index.php'</script>";
		}else{
		
			die('操作失败：'.mysql_error($conn));
		}
	}else{ //新增
		$sql = "INSERT INTO {$tabPre}gather_url(`gatherWebName`,`gatherWebCharset`,`gatherUrlType`,`gatherUrlSets`,`firstPageUrl`, `gatherUrl`,`startPage`,`endPage`,`subjectSeq`,`listUrlSeq`,`contentSeq`,`subjectPat`,`contentPat`,`subjectReplace`,`contentReplace`,`remotePicToLocal`, `allowSameSubject`, `subjectUrlMustInclude`, `subjectUrlNotInclude`, `contentUrlMustInclude`, `contentUrlNotInclude`, `dbInfo`, `subjectMustInclude` , `subjectNotInclude`, `contentMustInclude`, `contentNotInclude`, `subjectUseRealPat` , `subjectRealPat`, `contentUseRealPat`, `contentRealPat`) VALUES('$gatherWebName','$gatherWebCharset','$gatherUrlType','$gatherUrlSets','$firstPageUrl','$gatherUrl','$startPage','$endPage','$subjectSeq','$listUrlSeq','$contentSeq','$subjectPat','$contentPat','$subjectReplace','$contentReplace','$remotePicToLocal', '$allowSameSubject', '$subjectUrlMustInclude', '$subjectUrlNotInclude', '$contentUrlMustInclude', '$contentUrlNotInclude','$dbInfo', '$subjectMustInclude' , '$subjectNotInclude', '$contentMustInclude', '$contentNotInclude', '$subjectUseRealPat' , '$subjectRealPat', '$contentUseRealPat', '$contentRealPat')";
		if(mysql_query($sql, $conn)){
			echo "<script>alert('添加成功~');window.location='index.php'</script>";
		}else{
			die('操作失败：'.mysql_error($conn));
		}
	}

	
}
/**
 * 删除规则
 */
function deleteRule(){
	global $conn;
	global $tabPre;
	$id = intval($_GET['id']);
	if($id>0){
		$sql = "DELETE FROM  {$tabPre}gather_url WHERE id='$id' LIMIT 1 ;";
		if(mysql_query($sql)){
			echo "<script>alert('删除加成功~');window.location='index.php'</script>";
		}else{
			die('操作失败：'.mysql_error());
		}
	}
}

/**
 * 规则导出
 */
function export(){
	global $conn;
	global $tabPre;
	$id  = intval($_GET['id']);
	$sql = "SELECT * FROM {$tabPre}gather_url WHERE id='$id' LIMIT 1;";
	$res = mysql_query($sql,$conn);
	$row = mysql_fetch_assoc($res);
	if($row){
		//TO DO 字段名也自动读取，把id字段去掉
		$values      = "'";
		$values     .= implode("','", array_slice($row,1));
		$values     .= "'";
		$outFileName = iconv('UTF-8','GBK',$row['gatherWebName']);//防止中文乱码
		$exportSql   = "INSERT INTO {$tabPre}gather_url(`gatherWebName`,`gatherWebCharset`,`gatherUrlType`,`gatherUrlSets`,`gatherUrl`,`firstPageUrl`, `startPage`,`endPage`,`subjectSeq`,`listUrlSeq`,`contentSeq`,`subjectPat`,`contentPat`,`subjectReplace`,`contentReplace`,`remotePicToLocal`, `allowSameSubject`, `subjectUrlMustInclude`, `subjectUrlNotInclude`, `contentUrlMustInclude`, `contentUrlNotInclude`, `dbInfo`, `subjectMustInclude`, `subjectNotInclude`, `contentMustInclude`, `contentNotInclude`, `subjectUseRealPat`, `subjectRealPat`, `contentUseRealPat`, `contentRealPat`) VALUES($values)";
		 
		header('Content-type:application/file;');
		header('Content-Transfer-Encoding: binary');
		header('Pragma: public');
		header('Content-Disposition:attachment;filename='.$outFileName.'.txt');
		echo base64_encode($exportSql);
		die();
	}
}

/**
 * 导入规则
 */

function import(){
	global $conn;
	$data  = $_REQUEST['data'];
	$sql =  base64_decode($data);
	if($sql){
		$res = mysql_query($sql,$conn);
		if($res){
			p("<script>alert('导入规则成功!');location.href='index.php';</script>");
		}else{
			p(mysql_error($conn));
		}
	}
}
?>